import requests
import json
import re
from queue import Queue
from cou.utils.constants import PAGE_NUM
import threading
from pprint import pprint
from cou.models import *
from . import db
import time
from manage import app
class ele_spider():
    def __init__(self,phone_num,area_id=1):
        self.phone_num=phone_num
        self.my_account=requests.session()
        self.latitude='23.01755'
        self.longitude='113.414835'
        self.rank_id=None
        headers={"user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36",
                 "referer": "https://h5.ele.me/login/",
                 "cookie":"__wpkreporterwid_=4f44ee86-b369-4a28-947f-81cf3d8d499e; ubt_ssid=i5bxwaoogzzde9cbcsbxlum7eryvupd2_2020-09-14; perf_ssid=kyxo0wi7oxcxnt7if27ufqpc6x6vmt3k_2020-09-14; ut_ubt_ssid=o6cgkm8hjj4dxsf1nf2xv1jb0pqugris_2020-09-14; _bl_uid=baka3fey28s4FvnCtgear42syqXq; cna=+ZLgF+yIJx4CAdxzt4NA6vqN; _utrace=a8dc4ccab369fb8d7f61eced9bd8160d_2020-09-14; track_id=1600064076|da71caefac63546b7264a91a049d1f6f5817e1b4ad4d9fd695|d854ecccbc13e0f567062a616c4bba69; tzyy=4589a60d17bd55bad2738b6199cacd10; perf_ssid=iy9bu4vet5lg2a5b6f99wwrpp0jmm88r_2020-09-15; xlly_s=1; USERID=877938922; UTUSER=877938922; SID=DgAAAAA0VEjq-QAEAADuL8gEEwfqulsrdLFvVtnxTubH1iYxMglss78W; ZDS=1.0|1600433511|kmpJ7DB0tZbjBufOieMOG7je8KUcyNVq2YeH+/sIdVR74ZwmZ1cbQgFUyIRyX3OQfB5ZzS37DfU54Ef8iUHvbQ==; x5check_ele=V95QlPPIjFpHxSETGcEZVKlIneOC8XStIetp5mNybwI%3D; x5sec=7b227466653b32223a22323761303263323265333566353762633935613530353937323166656165626443492f596c767346454b5761754937342b654b7973674561437a67334e7a6b7a4f446b794d6a737a227d; isg=BIyMWcpqP7pPyyv8nywAtecGXeq-xTBvhZZyO-ZNmDfacSx7DtUA_4LHFX_JOWjH; tfstk=cEzOBNZ_49XMnV5IElInGUHKkT1lZTRtHLDHk_ps4c9WSALAi7zuyuilrXGKq6C..; l=eBjdqgseOz9MeYq8BOfwourza77OSIRAguPzaNbMiOCP9wCH54gRWZrW6vYMC3GRhsi2R3RZPRigBeYBqIcFaiP8EP17AkMmn"}
        self.my_account.headers=headers
        self.shop_list_url=Queue()
        self.shop_url=Queue()
        self.shop_list_data=Queue()
        self.proxy={'http':'http://221.6.187.12:4231'}
        self.shop_data=Queue()
        self.area_id=area_id
    def generate_shop_list_url(self):
        for i in range(PAGE_NUM):
            self.shop_list_url.put(i)
    def request_shop_list_url(self):
        params={
            "latitude": self.latitude ,
            "longitude": self.longitude         ,
            "keyword":  ""                     ,
            "offset":""                ,
            "limit": "8"                       ,
            "extras[]": ["activities","tags"]           ,
          
            "terminal": "h5"                   ,
            "brand_ids[]":""                   ,
            "restaurant_category_ids[]": ["770 ",
             "514",
             "1026",
             "1282",
             "263",
             "266",
             "778",
             "522",
             "1034",
             "1290",
             "267",
             "268",
             "269",
             "786",
             "530",
             "1042",
             "1298",
             "794",
             "538",
             "1050",
             "1306",
             "802",
             "546",
             "1058",
             "1314",
             "810",
             "554",
             "1066",
             "1322",
             "818",
             "562",
             "1074",
             "1330",
             "826",
             "570",
             "1082",
             "1338",
             "834",
             "578",
             "1090",
             "1346",
             "842",
             "586",
             "1098",
             "1354",
             "850",
             "594",
             "1106",
             "1362",
             "858",
             "602",
             "346",
             "1114",
             "354",
             "866",
             "610",
             "1122",
             "362",
             "874",
             "618",
             "1130",
             "370",
             "882",
             "626",
             "1138",
             "378",
             "890",
             "634",
             "1146",
             "386",
             "898",
             "642",
             "1154",
             "394",
             "906",
             "650",
             "1162",
             "402",
             "914",
             "658",
             "1170",
             "410",
             "922",
             "666",
             "1178",
             "418",
             "930",
             "674",
             "1186",
             "426",
             "938",
             "682",
             "1194",
             "434",
             "946",
             "690",
             "1202",
             "442",
             "954",
             "698",
             "1210",
             "450",
             "706",
             "962",
             "1218",
             "458",
             "970",
             "714",
             "1226",
             "209",
             "466",
             "978",
             "722",
             "1234",
             "212",
             "214",
             "474",
             "218",
             "986",
             "730",
             "221",
             "222",
             "223",
             "224",
             "225",
             "482",
             "226",
             "994",
             "738",
             "1250",
             "227",
             "228",
             "232",
             "490",
             "746",
             "234",
             "1002",
             "1258",
             "236",
             "240",
             "241",
             "498",
             "754",
             "1010",
             "242",
             "1266",
             "249",
             "762",
             "506",
             "1018",
             "250",
             "1274"]
        }
        while True:
            time.sleep(3)
            page=self.shop_list_url.get()
            params['offset']=str(page*8)
            request_url="https://h5.ele.me/restapi/shopping/v3/restaurants"
            res=self.my_account.get(request_url,params=params,proxies=self.proxy)
            self.shop_list_data.put(json.loads(res.content.decode()))
            self.shop_list_url.task_done()
    def parse_shop_list_data(self):
        while True:
           
            data1=self.shop_list_data.get()
            
            data=data1.get('items')
            if data:
                for i in data:

                    temp_shop=i['restaurant']

                    shop_id=temp_shop['id']
                    self.shop_url.put(shop_id)
                    # with app.app_context():
                    #     db.session.add(temp_shop_model)
                    #     db.session.commit()
            else:
                pprint(data1)
            self.shop_list_data.task_done()
    def request_shop_url(self):
        while True:
            time.sleep(3)
            id=self.shop_url.get()
            params={'terminal': 'h5','user_id':self.my_account.cookies.get('USERID'),'extras': ["activities","albums","license","identification","qualification"],'latitude': self.latitude,'longitude': self.longitude}
            requests_url="https://h5.ele.me/pizza/shopping/restaurants/{}/batch_shop".format(id)
            requests_url2="https://h5.ele.me/restapi/shopping/v1/restaurants/{}/exclusive_hongbao/detail".format(id)
            params2={'terminal': 'h5','user_id':self.my_account.cookies.get('USERID'),'latitude': self.latitude,'longitude': self.longitude}
            res2=self.my_account.get(requests_url2,params=params2,proxies=self.proxy)
            res=self.my_account.get(requests_url,params=params,proxies=self.proxy)
            data=json.loads(res.content.decode())
            data['redpack']=json.loads(res2.content.decode())
            self.shop_data.put(data)
            self.shop_url.task_done()
    def parse_shop_data(self):
        while True:
            with app.app_context():
                data1=self.shop_data.get()
                data=data1.get('rst')
                
                if data:
                    name=data['name']
                    id=data['id']
                    try:
                        
                        temp_shop=Shop.query.filter_by(name=name,area_id=self.area_id).first()
                    except Exception as e:
                        print(e)
                    else:
                        flag=1
                        if not temp_shop:
                            temp_shop=Shop()
                            temp_shop.shop_id=id
                            flag=0
                        temp_shop.name=name
                        temp_shop.area_id=self.area_id
                       
                        if not flag:
                            db.session.add(temp_shop)
                        db.session.commit()
                        
                        discounts=data.get('activities')
                            
                        Manjian.query.filter_by(shop_id=temp_shop.id).delete()
                        Juan.query.filter_by(shop_id=temp_shop.id).delete()
                        db.session.commit()
                        print("正在爬取",temp_shop.id,"的满减信息")
                        if discounts:
                            for i in discounts:
                                if i['type']==102 or i['type']=='102':
                                    
                                    discounts=re.findall("满(\d+)元减(\d+)元",i['tips'])
                                    for j in discounts:
                                        man=int(float(j[0])*100)
                                        jian=int(float(j[1])*100)
                                        temp_manjian=Manjian()
                                        temp_manjian.shop_id=temp_shop.id
                                        temp_manjian.man=man
                                        temp_manjian.jian=jian
                                        db.session.add(temp_manjian)
                                        db.session.commit()
                        print(temp_shop.id,"的满减信息爬取完毕")
                        temp_menu=data1.get('menu')
                        Food.query.filter_by(shop_id=temp_shop.id).delete()
                        db.session.commit()
                        print("正在爬取",temp_shop.id,"的商品信息")
                        for i in temp_menu:
                            temp_foods=i['foods']
                            for j in temp_foods:
                                if j['activity']!=None:
                                    #print (j['activity'])
                                    continue
                                for k in j['specfoods']:
                                    temp_packing_fee=int(float(k['packing_fee'])*100)
                                    temp_price=int(float(k['price'])*100)
                                    temp_name=k['name']
                                    temp_food=Food()
                                    temp_food.name=temp_name
                                    temp_food.price=temp_price+temp_packing_fee
                                    temp_food.shop_id=temp_shop.id
                                    temp_food.url="https://cube.elemecdn.com/a/7c/a7e9e5aa15b1b8fc6f1bece8ee385jpeg.jpeg"
                                    check_food=Food.query.filter_by(shop_id=temp_shop.id,price=temp_price,name=temp_name).first()
                                    if not check_food:
                                        db.session.add(temp_food)
                                        db.session.commit()
                        print(temp_shop.id,"的商品信息爬取完毕")
                        redpack=data1['redpack']
                        print("正在爬取",temp_shop.id,"的红包信息")
                        for i in redpack:
                            jian=int(float(i['amount'])*100)
                            man= re.search("\d+(\.\d+)?",i['name'])
                            
                            temp_juan=Juan()
                            temp_juan.jian=jian
                            temp_juan.man=int(float(man.group())*100) if man else 0
                            if i['condition_type']==1:
                                temp_juan.is_huiyuanjuan=1
                                temp_juan.huiyuan_price=jian
                            else:
                                temp_juan.is_huiyuanjuan=0
                            temp_juan.shop_id=temp_shop.id
                            db.session.add(temp_juan)
                            db.session.commit()
                        print(temp_shop.id,"的红包信息爬取完毕")
                else:
                    pprint(data1)


            self.shop_data.task_done()

            

          
            





    def run2(self):
        with app.app_context():
            Shops=Shop.query.all()
            for i in Shops[1:-1]:
                shop_id=i.shop_id
                self.shop_url.put(shop_id)
            thread_list=[]
            for i in range(1):
                thr=threading.Thread(target=self.request_shop_url)
                thread_list.append(thr)
            for i in range(4):
                thr=threading.Thread(target=self.parse_shop_data)
                thread_list.append(thr)
            for i in thread_list:
                i.setDaemon(True)
                i.start()
            
            self.shop_url.join()
            
            self.shop_data.join()
        
            print("主线程结束")
        
            

            

    def test(self):
        a=Area()
        a.name="暨南大学石牌校区"
        db.session.add(a)
        db.session.commit()    
    def run(self):
        thread_list=[]
        for i in range(1):
            thr=threading.Thread(target=self.generate_shop_list_url)
            thread_list.append(thr)
        for i in range(1):
            thr=threading.Thread(target=self.request_shop_list_url)
            thread_list.append(thr)
            
        for i in range(4):
            thr=threading.Thread(target=self.parse_shop_list_data)
            thread_list.append(thr)
            
        
        for i in range(1):
            thr=threading.Thread(target=self.request_shop_url)
            thread_list.append(thr)
        for i in range(4):
            thr=threading.Thread(target=self.parse_shop_data)
            thread_list.append(thr)
        for i in thread_list:
            i.setDaemon(True)
            i.start()
        
        self.shop_url.join()
        
        self.shop_data.join()
        self.shop_list_url.join()
        self.shop_list_data.join()
        
        print("主线程结束")
if __name__ =="__main__":
    # a=ele_spider("13048704663")
    # a.run()
    a=Taocan()
            

    




